
import os
from IPython.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the next line of this cell.
# antenna = "004"
# csv_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/_rtp_summary_'
# auto_metrics_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/auto_metrics_inspect'
# os.environ["ANTENNA"] = antenna
# os.environ["CSV_FOLDER"] = csv_folder
# os.environ["AUTO_METRICS_FOLDER"] = auto_metrics_folder


# Use environment variables to figure out path to the csvs and auto_metrics
antenna = str(int(os.environ["ANTENNA"]))
csv_folder = os.environ["CSV_FOLDER"]
auto_metrics_folder = os.environ["AUTO_METRICS_FOLDER"]
print(f'antenna = "{antenna}"')
print(f'csv_folder = "{csv_folder}"')
print(f'auto_metrics_folder = "{auto_metrics_folder}"')

antenna = "143"
csv_folder = "/home/obs/src/H6C_Notebooks/_rtp_summary_"
auto_metrics_folder = "/home/obs/src/H6C_Notebooks/auto_metrics_inspect"


display(HTML(f'<h1 style=font-size:50px><u>Antenna {antenna} Report</u><p></p></h1>'))


import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import glob
import re
from hera_notebook_templates.utils import status_colors, Antenna


# load csvs and auto_metrics htmls in reverse chronological order
csvs = sorted(glob.glob(os.path.join(csv_folder, 'rtp_summary_table*.csv')))[::-1]
print(f'Found {len(csvs)} csvs in {csv_folder}')
auto_metric_htmls = sorted(glob.glob(auto_metrics_folder + '/auto_metrics_inspect_*.html'))[::-1]
print(f'Found {len(auto_metric_htmls)} auto_metrics notebooks in {auto_metrics_folder}')

Found 23 csvs in /home/obs/src/H6C_Notebooks/_rtp_summary_
Found 21 auto_metrics notebooks in /home/obs/src/H6C_Notebooks/auto_metrics_inspect


# Per-season options
mean_round_modz_cut = 4
dead_cut = 0.4
crossed_cut = 0.0

def jd_to_summary_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/_rtp_summary_/rtp_summary_{jd}.html'

def jd_to_auto_metrics_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/auto_metrics_inspect/auto_metrics_inspect_{jd}.html'


this_antenna = None
jds = []

# parse information about antennas and nodes
for csv in csvs:
    df = pd.read_csv(csv)
    for n in range(len(df)):
        # Add this day to the antenna
        row = df.loc[n]
        if isinstance(row['Ant'], str) and '<a href' in row['Ant']:
            antnum = int(row['Ant'].split('</a>')[0].split('>')[-1]) # it's a link, extract antnum
        else:
            antnum = int(row['Ant'])
        if antnum != int(antenna):
            continue
        
        if np.issubdtype(type(row['Node']), np.integer):
            row['Node'] = str(row['Node'])
        if type(row['Node']) == str and row['Node'].isnumeric():
            row['Node'] = 'N' + ('0' if len(row['Node']) == 1 else '') + row['Node']
            
        if this_antenna is None:
            this_antenna = Antenna(row['Ant'], row['Node'])
        jd = [int(s) for s in re.split('_|\.', csv) if s.isdigit()][-1]
        jds.append(jd)
        this_antenna.add_day(jd, row)
        break


# build dataframe
to_show = {'JDs': [f'<a href="{jd_to_summary_url(jd)}" target="_blank">{jd}</a>' for jd in jds]}
to_show['A Priori Status'] = [this_antenna.statuses[jd] for jd in jds]

df = pd.DataFrame(to_show)

# create bar chart columns for flagging percentages:
bar_cols = {}
bar_cols['Auto Metrics Flags'] = [this_antenna.auto_flags[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jee)'] = [this_antenna.dead_flags_Jee[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jnn)'] = [this_antenna.dead_flags_Jnn[jd] for jd in jds]
bar_cols['Crossed Fraction in Ant Metrics'] = [this_antenna.crossed_flags[jd] for jd in jds]
bar_cols['Flag Fraction Before Redcal'] = [this_antenna.flags_before_redcal[jd] for jd in jds]
bar_cols['Flagged By Redcal chi^2 Fraction'] = [this_antenna.redcal_flags[jd] for jd in jds]
for col in bar_cols:
    df[col] = bar_cols[col]

z_score_cols = {}
z_score_cols['ee Shape Modified Z-Score'] = [this_antenna.ee_shape_zs[jd] for jd in jds]
z_score_cols['nn Shape Modified Z-Score'] = [this_antenna.nn_shape_zs[jd] for jd in jds]
z_score_cols['ee Power Modified Z-Score'] = [this_antenna.ee_power_zs[jd] for jd in jds]
z_score_cols['nn Power Modified Z-Score'] = [this_antenna.nn_power_zs[jd] for jd in jds]
z_score_cols['ee Temporal Variability Modified Z-Score'] = [this_antenna.ee_temp_var_zs[jd] for jd in jds]
z_score_cols['nn Temporal Variability Modified Z-Score'] = [this_antenna.nn_temp_var_zs[jd] for jd in jds]
z_score_cols['ee Temporal Discontinuties Modified Z-Score'] = [this_antenna.ee_temp_discon_zs[jd] for jd in jds]
z_score_cols['nn Temporal Discontinuties Modified Z-Score'] = [this_antenna.nn_temp_discon_zs[jd] for jd in jds]
for col in z_score_cols:
    df[col] = z_score_cols[col]

ant_metrics_cols = {}
ant_metrics_cols['Average Dead Ant Metric (Jee)'] = [this_antenna.Jee_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Dead Ant Metric (Jnn)'] = [this_antenna.Jnn_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Crossed Ant Metric'] = [this_antenna.crossed_metrics[jd] for jd in jds]
for col in ant_metrics_cols:
    df[col] = ant_metrics_cols[col]

redcal_cols = {}
redcal_cols['Median chi^2 Per Antenna (Jee)'] = [this_antenna.Jee_chisqs[jd] for jd in jds]
redcal_cols['Median chi^2 Per Antenna (Jnn)'] = [this_antenna.Jnn_chisqs[jd] for jd in jds]   
for col in redcal_cols:
    df[col] = redcal_cols[col]

# style dataframe
table = df.style.hide_index()\
          .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
          .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=list(z_score_cols.keys())) \
          .background_gradient(cmap='bwr_r', vmin=dead_cut-.25, vmax=dead_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .background_gradient(cmap='bwr_r', vmin=crossed_cut-.25, vmax=crossed_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .background_gradient(cmap='plasma', vmax=4, vmin=1, axis=None, subset=list(redcal_cols.keys())) \
          .applymap(lambda val: 'font-weight: bold' if val < dead_cut else '', subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val < crossed_cut else '', subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .applymap(lambda val: 'color: red' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .bar(subset=list(bar_cols.keys()), vmin=0, vmax=1) \
          .format({col: '{:,.4f}'.format for col in z_score_cols}) \
          .format({col: '{:,.4f}'.format for col in ant_metrics_cols}) \
          .format('{:,.2%}', na_rep='-', subset=list(bar_cols.keys())) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])


display(HTML(f'<h2>Antenna {antenna}, Node {this_antenna.node}:</h2>'))
HTML(table.render(render_links=True, escape=False))


htmls_to_display = []
for am_html in auto_metric_htmls:
    html_to_display = ''
    # read html into a list of lines
    with open(am_html) as f:
        lines = f.readlines()
    
    # find section with this antenna's metric plots and add to html_to_display
    jd = [int(s) for s in re.split('_|\.', am_html) if s.isdigit()][-1]
    try:
        section_start_line = lines.index(f'<h2>Antenna {antenna}: {jd}</h2>\n')
    except ValueError:
        continue
    html_to_display += lines[section_start_line].replace(str(jd), f'<a href="{jd_to_auto_metrics_url(jd)}" target="_blank">{jd}</a>')
    for line in lines[section_start_line + 1:]:
        html_to_display += line
        if '<hr' in line:
            htmls_to_display.append(html_to_display)
            break


for i, html_to_display in enumerate(htmls_to_display):
    if i == 100:
        break
    display(HTML(html_to_display))

JDs	A Priori Status	Auto Metrics Flags	Dead Fraction in Ant Metrics (Jee)	Dead Fraction in Ant Metrics (Jnn)	Crossed Fraction in Ant Metrics	Flag Fraction Before Redcal	Flagged By Redcal chi^2 Fraction	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	Average Dead Ant Metric (Jee)	Average Dead Ant Metric (Jnn)	Average Crossed Ant Metric	Median chi^2 Per Antenna (Jee)	Median chi^2 Per Antenna (Jnn)
2459838	digital_ok	0.00%	0.00%	0.00%	0.00%	0.61%	0.00%	0.107570	-0.849127	0.949094	-0.176619	0.653057	-0.634357	-0.635403	-0.909537	0.7632	0.7455	0.3742	2.141233	1.716652
2459836	digital_ok	-	0.00%	0.00%	0.00%	-	-	nan	nan	nan	nan	nan	nan	nan	nan	0.7150	0.6389	0.4592	nan	nan
2459835	digital_ok	100.00%	0.00%	0.00%	0.00%	-	-	4.986194	5.346482	9.859963	11.046389	39.365519	20.842917	-2.496708	-4.880729	0.7975	0.5080	0.6135	nan	nan
2459833	digital_ok	100.00%	0.00%	0.00%	0.00%	-	-	18.142989	19.128813	25.466486	30.655857	337.534161	294.871847	52.893010	38.391197	0.7960	0.4992	0.6023	nan	nan
2459832	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	0.00%	-0.208412	-1.230586	0.892922	-0.403340	0.533688	-0.662260	-0.081521	-1.217264	0.8124	0.5751	0.5551	1.566945	1.146094
2459831	digital_ok	0.00%	100.00%	100.00%	0.00%	-	-	-0.668577	-0.658463	2.171867	0.894179	1.384778	1.140078	0.342314	0.016924	0.0268	0.0262	0.0008	nan	nan
2459830	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	0.00%	0.408918	-1.165728	1.420167	0.151858	0.641596	-0.830831	-0.232292	-1.611400	0.8110	0.5828	0.5446	1.485808	1.069551
2459829	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	99.35%	1.098176	-0.654129	1.142783	0.576182	0.712272	-1.207323	-0.190796	-1.080284	0.7604	0.7018	0.4005	31.250365	5.744279
2459828	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	100.00%	-0.348696	-1.033064	0.854497	0.702458	0.338614	-0.404753	-0.288882	-0.910805	0.8104	0.5931	0.5264	4.103826	3.374512
2459827	digital_ok	0.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.229226	-0.642359	1.261196	1.428152	-0.030140	-1.181060	-0.527700	-0.490320	0.7695	0.7103	0.3985	0.000000	0.000000
2459826	digital_ok	0.00%	0.00%	0.00%	0.00%	92.11%	7.89%	-0.770523	-0.755588	1.395475	1.177486	0.168586	-0.467078	-0.235727	-1.342941	0.8135	0.6282	0.4854	0.000000	0.000000
2459825	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	-0.586639	-0.826270	1.426261	-0.191591	6.175496	5.407548	5.081687	5.060673	0.8125	0.6348	0.4889	0.000000	0.000000
2459824	digital_ok	0.00%	0.00%	0.00%	0.00%	91.00%	8.00%	1.023329	-1.192953	1.805892	-0.238090	2.006341	0.946278	2.987605	2.249181	0.7420	0.7704	0.3440	0.000000	0.000000
2459823	digital_ok	0.00%	0.00%	0.00%	0.00%	100.00%	0.00%	-0.276781	-1.082827	1.421349	1.067012	0.415518	-0.003740	0.229084	-1.728416	0.7815	0.6821	0.4453	0.000000	0.000000
2459822	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	0.00%	-0.122048	-1.281522	1.181881	1.118058	0.288319	-0.797625	0.790964	-0.016499	0.8225	0.6630	0.4811	1.054050	0.819413
2459821	digital_ok	0.00%	0.00%	0.00%	0.00%	60.53%	7.89%	-0.024679	-1.452225	1.105030	1.332077	1.121858	0.140068	-0.525399	-0.545795	0.8091	0.6601	0.4886	0.000000	0.000000
2459820	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	0.53%	0.689771	-1.300966	0.942541	1.361010	-0.642433	-1.447060	-0.556075	-1.028992	0.7835	0.7110	0.3936	0.933903	0.749753
2459817	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	21.05%	0.020239	-1.595185	1.032667	1.106298	0.905571	0.283526	0.968178	0.723467	0.8179	0.6936	0.4813	2.025096	1.608779
2459816	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	2.33%	0.326186	-1.138979	2.118010	-0.017114	1.030465	-0.915315	-0.286297	-1.277737	0.8470	0.6306	0.5561	1.837578	1.535315
2459815	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	0.00%	0.117660	-1.457066	1.591755	0.415509	1.212045	0.156415	-0.158918	-1.127915	0.8100	0.6981	0.5014	1.577629	1.238277
2459814	digital_ok	0.00%	-	-	-	-	-	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan
2459813	digital_ok	0.00%	0.00%	0.00%	0.00%	80.21%	0.00%	1.063115	-1.307812	0.542583	1.010056	-0.097266	-1.259205	-0.321902	-0.864702	0.7980	0.7286	0.3998	0.000000	0.000000

Antenna Season Report Notebook¶

Antenna 143 Report

Load relevant info from summary CSVs¶

Table 1: Per-Night RTP Summary Info For This Atenna¶

Antenna 143, Node N14:

Load antenna metric spectra and waterfalls from `auto_metrics` notebooks.¶

Figure 1: Antenna autocorrelation metric spectra and waterfalls.¶

Antenna 143: 2459838

Antenna 143: 2459835

Antenna 143: 2459833

Antenna 143: 2459832

Antenna 143: 2459831

Antenna 143: 2459830

Antenna 143: 2459829

Antenna 143: 2459828

Antenna 143: 2459827

Antenna 143: 2459826

Antenna 143: 2459825

Antenna 143: 2459824

Antenna 143: 2459823

Antenna 143: 2459822

Antenna 143: 2459821

Antenna 143: 2459820

Antenna 143: 2459817

Antenna 143: 2459816

Antenna 143: 2459815

Antenna 143: 2459814

Antenna 143: 2459813

Antenna Season Report Notebook¶

Antenna 143 Report

Load relevant info from summary CSVs¶

Table 1: Per-Night RTP Summary Info For This Atenna¶

Antenna 143, Node N14:

Load antenna metric spectra and waterfalls from auto_metrics notebooks.¶

Figure 1: Antenna autocorrelation metric spectra and waterfalls.¶

Antenna 143: 2459838

Antenna 143: 2459835

Antenna 143: 2459833

Antenna 143: 2459832

Antenna 143: 2459831

Antenna 143: 2459830

Antenna 143: 2459829

Antenna 143: 2459828

Antenna 143: 2459827

Antenna 143: 2459826

Antenna 143: 2459825

Antenna 143: 2459824

Antenna 143: 2459823

Antenna 143: 2459822

Antenna 143: 2459821

Antenna 143: 2459820

Antenna 143: 2459817

Antenna 143: 2459816

Antenna 143: 2459815

Antenna 143: 2459814

Antenna 143: 2459813

Load antenna metric spectra and waterfalls from `auto_metrics` notebooks.¶